import scrapy
from scrapy import Request
from utils.jobbossstrUtil import  *
from utils.dbUtils import DataBase
import time

class contentSpider(scrapy.Spider):
    name = "bossjobSpider"
    # allowed_domains = ["jobs.51job.com"]
    prefix_url = 'https://www.zhipin.com'

    job_url='https://www.zhipin.com/c100010000/?query=java&period=1&page={page}&ka=page-{page}'

    company_sql = 'insert into bossjob_company (COMPANY_NAME,CITY,AREA,ADDRESS,NATURE,SCALE,BUSINESS,COMPANY_URL,CREATE_TIME)' \
                    ' values (%s,%s,%s,%s,%s,%s,%s,%s,now())'

    company_search_sql = 'select id from  bossjob_company where COMPANY_NAME=%s and COMPANY_URL=%s '

    jobrecord_sql = 'insert into bossjob_record(COMPANY_ID,JOB_NAME,MIN_SALARY,MAX_SALARY,JOB_EXPERIENCE,EDUCATION,RECRUITING_NUMBERS,JOB_INFORMATION,RELEASE_TIME,CREATE_TIME)'\
                    'values (%s,%s,%s,%s,%s,%s,%s,%s,%s,now())'
    db = DataBase()

    def start_requests(self):
        yield Request(self.job_url.format(page=1), callback=self.parse,meta={'page':1},dont_filter=True)

    def parse(self, response):
        divs = response.xpath('//div[@class="job-primary"]')
        for div in divs:
            meta = {}
            job_url = self.prefix_url + div.xpath('div[@class="info-primary"]/h3/a/@href').extract()[0]
            meta['jobname'] = div.xpath('div[@class="info-primary"]/h3/a/div[@class="job-title"]/text()').extract()[0]
            meta['salary'] = div.xpath('div[@class="info-primary"]/h3/a/span[@class="red"]/text()').extract()[0]
            meta['area'] = div.xpath('div[@class="info-primary"]/p/text()[1]').extract()[0]# 地区
            meta['experience']  = div.xpath('div[@class="info-primary"]/p/text()[2]').extract()[0]# 工作经验
            meta['education']  = div.xpath('div[@class="info-primary"]/p/text()[3]').extract()[0]# 学历
            meta['company_name']  = div.xpath('div[@class="info-company"]/div/h3/a/text()').extract()[0]# 公司名称
            meta['company_url']  = self.prefix_url+div.xpath('div[@class="info-company"]/div/h3/a/@href').extract()[0]  # 公司url
            meta['business']  = div.xpath('div[@class="info-company"]/div/p/text()[1]').extract()[0]  # 公司业务
            meta['nature'] = div.xpath('div[@class="info-company"]/div/p/text()[2]').extract()[0]  # 公司状态
            meta['scale'] = div.xpath('div[@class="info-company"]/div/p/text()[3]').extract()[0]  # 规模
            yield Request(url=job_url, callback=self.parse_job, meta=meta, dont_filter=True)


    def parse_job(self, response):
        meta = response.meta
        company_name = meta.get('company_name')
        company_url = meta.get('company_url')
        jobname = meta.get('jobname')
        area = meta.get('area')
        business = meta.get('business')
        experience = meta.get('experience')
        education = meta.get('education')
        nature = meta.get('nature')
        scale = meta.get('scale')
        salary = meta.get('salary')
        adreess = ''
        job_detail = ''
        try:
            adreess = response.xpath('//div[@class="location-address"]/text()').extract()[0]
            job_detail_div = response.xpath('//div[@class="detail-content"]/div/div')
            job_detail = job_detail_div.xpath('string(.)').extract()[0].strip()
        except Exception as e:
            pass
        companyDetail = []
        companyDetail.append(company_name)
        companyDetail = companyDetail+area_deal(area)
        companyDetail.append(adreess)
        companyDetail.append(business)
        companyDetail.append(nature)
        companyDetail.append(scale)
        companyDetail.append(company_url)
        company_search_condition = []
        company_search_condition.append(company_name)
        company_search_condition.append(company_url)
        record = self.db.select_one(self.company_search_sql,tuple(company_search_condition))
        if record is None:
            #插入
            company_id = self.db.insert_one(self.company_sql,tuple(companyDetail))
        else:
            company_id = record[0]

        jobresult = []
        jobresult.append(company_id)
        jobresult.append(jobname)
        jobresult += salary_deal(salary)
        jobresult.append(experience)
        jobresult.append(education)
        jobresult.append('')#招聘人数 无
        jobresult.append(job_detail)
        jobresult.append(time.strftime("%Y-%m-%d"))
        self.db.insert_one(self.jobrecord_sql, tuple(jobresult))
